import { createCrawl,createCrawlOpenAI } from 'x-crawl'
import dotenv from 'dotenv'

dotenv.config()


console.log(process.env.OPENAI_API_KEY);
console.log(process.env.OPEN_BASE_URL)

//创建爬虫
const crawlApp = createCrawl({
    maxRetry:3,
    intervalTime:{min:1000,max:3000},
})
//创建OpenAI爬虫
const crawlOpenAI = createCrawlOpenAI({
    clientOptions:{
        apiKey: process.env.OPENAI_API_KEY,
        baseUrl: process.env.OPEN_BASE_URL,
    },
    defaultModel:{chatModel:'openai/gpt-4o'}
})


crawlApp.crawlPage('http://movie.douban.com/chart').then(async (res)=>{
    const {page,browser} = res.data

    //获取页面内容
    const targetSelector = '.article'
    await page.waitForSelector(targetSelector)
    const content = await page.$eval(targetSelector,(el)=>el.innerText)
    //让ai解析出我们要的数据
    const result = await crawlOpenAI.parseElements(content,
    `获取电影评分,将评分不小于8.0的电影的图片链接、电影名称、电影上映时间获取到。
    输出格式为JSON格式。如:
    [
        {
            "image": "https://img3.doubanio.com/view/photo/s_ratio_poster/public/p2561649582.jpg",
            "name": "肖申克的救赎",
            "time": "1994-09-10"
        }
    ]`
    )
    browser.close()
    console.log(result);

})
